In [1]:
%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

plt.rcParams['figure.figsize'] = (50.0, 60.0)
In [2]:
# Importing the dataset
df = pd.read_csv("/home/z/Python/CiscoAML/data/nids-aml-table.csv", header=0)
In [3]:
df.head()
Out[3]:
Feature CTU-13 RF CTU-13 Boruta CICIDS17 RF CICIDS17 Boruta NIDS-AML Auto RF NIDS-AML Auto Boruta NIDS-AML Human RF NIDS-AML Human Boruta Unnamed: 9
0 Src Port 0.01870 x 0.02925 NaN 0.00344 x 0.00809 NaN NaN
1 Dst Port 0.03436 x 0.05697 x 0.05814 x 0.00271 NaN NaN
2 Protocol 0.00082 x 0.01640 NaN 0.00001 NaN 0.00419 NaN NaN
3 Timestamp 0.32922 x 0.07393 x 0.15306 x 0.02646 NaN NaN
4 Flow Duration 0.01564 NaN 0.00183 NaN 0.01140 x 0.02523 NaN NaN

Processing

In [4]:
df = df.fillna(0)
In [5]:
df.round(decimals=3)
Out[5]:
Feature CTU-13 RF CTU-13 Boruta CICIDS17 RF CICIDS17 Boruta NIDS-AML Auto RF NIDS-AML Auto Boruta NIDS-AML Human RF NIDS-AML Human Boruta Unnamed: 9
0 Src Port 0.019 x 0.029 0 0.003 x 0.008 0.0 0.0
1 Dst Port 0.034 x 0.057 x 0.058 x 0.003 0.0 0.0
2 Protocol 0.001 x 0.016 0 0.000 0 0.004 0.0 0.0
3 Timestamp 0.329 x 0.074 x 0.153 x 0.026 0.0 0.0
4 Flow Duration 0.016 0 0.002 0 0.011 x 0.025 0.0 0.0
... ... ... ... ... ... ... ... ... ... ...
75 Active Min 0.000 0 0.000 0 0.000 0 0.000 0.0 0.0
76 Idle Mean 0.120 0 0.001 0 0.058 x 0.035 0.0 0.0
77 Idle Std 0.001 x 0.000 0 0.000 0 0.003 0.0 0.0
78 Idle Max 0.131 x 0.002 0 0.024 x 0.068 0.0 0.0
79 Idle Min 0.034 x 0.003 0 0.063 x 0.062 0.0 0.0

80 rows × 10 columns

In [6]:
df = df.replace(to_replace=r'x', value=0.35, regex=True)
In [7]:
df = df.drop(columns='Unnamed: 9')
In [8]:
print(df.columns)
Index(['Feature', 'CTU-13 RF', 'CTU-13 Boruta', 'CICIDS17 RF',
       'CICIDS17 Boruta', 'NIDS-AML Auto RF', 'NIDS-AML Auto Boruta',
       'NIDS-AML Human RF', 'NIDS-AML Human Boruta'],
      dtype='object')
In [9]:
df_labels = df.Feature
In [10]:
df_data = df.drop(columns='Feature')
In [11]:
arr = df.drop(columns=['Feature'])
In [12]:
print(type(arr))
<class 'pandas.core.frame.DataFrame'>
In [13]:
arr = arr.squeeze()
In [14]:
def heatmap(data, row_labels, col_labels, ax=None,
            cbar_kw={}, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Parameters
    ----------
    data
        A 2D numpy array of shape (N, M).
    row_labels
        A list or array of length N with the labels for the rows.
    col_labels
        A list or array of length M with the labels for the columns.
    ax
        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If
        not provided, use current axes or create a new one.  Optional.
    cbar_kw
        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.
    cbarlabel
        The label for the colorbar.  Optional.
    **kwargs
        All other arguments are forwarded to `imshow`.
    """

    if not ax:
        ax = plt.gca()

    # Plot the heatmap
    im = ax.imshow(data, **kwargs)

    # Create colorbar
    cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)
    cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom")

    # We want to show all ticks...
    ax.set_xticks(np.arange(data.shape[1]))
    ax.set_yticks(np.arange(data.shape[0]))
    # ... and label them with the respective list entries.
    ax.set_xticklabels(col_labels)
    ax.set_yticklabels(row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=-30, ha="right",
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    for edge, spine in ax.spines.items():
        spine.set_visible(False)

    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)
    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.tick_params(which="minor", bottom=False, left=False)

    return im, cbar


def annotate_heatmap(im, data=None, valfmt="{x:.4f}",
                     textcolors=["black", "white"],
                     threshold=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A list or array of two color specifications.  The first is used for
        values below a threshold, the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max())/2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

Matplotlib Annotated Heatmap With Boruta

In [15]:
feature = ['Src Port', 'Dst Port', 'Protocol', 'Timestamp','Flow Duration', 'Total Fwd Packet',
       'Total Bwd packets', 'Total Length of Fwd Packet',
       'Total Length of Bwd Packet', 'Fwd Packet Length Max',
       'Fwd Packet Length Min', 'Fwd Packet Length Mean',
       'Fwd Packet Length Std', 'Bwd Packet Length Max',
       'Bwd Packet Length Min', 'Bwd Packet Length Mean',
       'Bwd Packet Length Std', 'Flow Bytes/s', 'Flow Packets/s','Flow IAT Mean', 'Flow IAT Std',
       'Flow IAT Max', 'Flow IAT Min', 'Fwd IAT Total', 'Fwd IAT Mean',
       'Fwd IAT Std', 'Fwd IAT Max', 'Fwd IAT Min', 'Bwd IAT Total',
       'Bwd IAT Mean', 'Bwd IAT Std', 'Bwd IAT Max', 'Bwd IAT Min',
       'Fwd PSH Flags', 'Bwd PSH Flags', 'Fwd URG Flags', 'Bwd URG Flags',
       'Fwd Header Length', 'Bwd Header Length', 'Fwd Packets/s',
       'Bwd Packets/s', 'Packet Length Min', 'Packet Length Max',
       'Packet Length Mean', 'Packet Length Std', 'Packet Length Variance',
       'FIN Flag Count', 'SYN Flag Count', 'RST Flag Count', 'PSH Flag Count',
       'ACK Flag Count', 'URG Flag Count', 'CWE Flag Count', 'ECE Flag Count',
       'Down/Up Ratio', 'Average Packet Size', 'Fwd Segment Size Avg',
       'Bwd Segment Size Avg', 'Fwd Bytes/Bulk Avg', 'Fwd Packet/Bulk Avg',
       'Fwd Bulk Rate Avg', 'Bwd Bytes/Bulk Avg', 'Bwd Packet/Bulk Avg',
       'Bwd Bulk Rate Avg', 'Subflow Fwd Packets', 'Subflow Fwd Bytes',
       'Subflow Bwd Packets', 'Subflow Bwd Bytes', 'FWD Init Win Bytes',
       'Bwd Init Win Bytes', 'Fwd Act Data Pkts', 'Fwd Seg Size Min',
       'Active Mean', 'Active Std', 'Active Max', 'Active Min', 'Idle Mean',
       'Idle Std', 'Idle Max', 'Idle Min']

algo = ['CTU-13 RF', 'CTU-13 Boruta', 'CICIDS17 RF', 'CICIDS17 Boruta', 'NIDS-AML Auto RF', 'NIDS-AML Auto Boruta',
       'NIDS-AML Human RF', 'NIDS-AML Human Boruta']

finding = arr

fig, ax = plt.subplots()

im, cbar = heatmap(finding, feature, algo, ax=ax,
                   cmap="magma_r", cbarlabel="higher correlation")
texts = annotate_heatmap(im, valfmt="{x:.4f}")

ax.axes.set_title("Feature Correlation Across NIDS Datasets",fontsize=20)
ax.set_xlabel("Score",fontsize=10)
ax.set_ylabel("Category",fontsize=10)

plt.show()

Seaborn Annotated Heatmap

In [16]:
import seaborn as sns

sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df_data, yticklabels=y_axis_labels, 
                cmap='magma_r', annot=True, fmt=".4f",annot_kws={'size':25})

Removing Boruta and Graphing

In [17]:
df2 = df.drop(columns=['CTU-13 Boruta', 'CICIDS17 Boruta', 'NIDS-AML Auto Boruta', 'NIDS-AML Human Boruta'])
In [18]:
df2 = df2.drop(columns=['Feature'])
In [19]:
df2.head()
Out[19]:
CTU-13 RF CICIDS17 RF NIDS-AML Auto RF NIDS-AML Human RF
0 0.01870 0.02925 0.00344 0.00809
1 0.03436 0.05697 0.05814 0.00271
2 0.00082 0.01640 0.00001 0.00419
3 0.32922 0.07393 0.15306 0.02646
4 0.01564 0.00183 0.01140 0.02523
In [20]:
sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df2, yticklabels=y_axis_labels, 
                cmap='magma_r', annot=True, fmt=".4f",annot_kws={'size':25})
In [21]:
print(type(df2))
<class 'pandas.core.frame.DataFrame'>
In [22]:
from sklearn import preprocessing

Using 'max' normalization

x_normalized = x / max(x)

In [23]:
df2_norm_max = preprocessing.normalize(df2, norm='max')

sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df2_norm_max, yticklabels=y_axis_labels, 
                cmap='magma_r', annot=True, fmt=".4f",annot_kws={'size':25})

Using 'l1' normalization

x_normalized = x / sum(X)

In [24]:
df2_norm_l1 = preprocessing.normalize(df2, norm='l1')

sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df2_norm_l1, yticklabels=y_axis_labels, 
                cmap='magma_r', annot=True, fmt=".4f",annot_kws={'size':25})

Using 'l2' normalization

x_normalized = x / sqrt(sum((i**2) for i in X))

In [26]:
df2_norm_l2 = preprocessing.normalize(df2, norm='l1')

sns.set(font_scale=2)

y_axis_labels = feature

# create seaborn heatmap with required label
p = sns.heatmap(df2_norm_l2, yticklabels=y_axis_labels, 
                cmap='magma_r', annot=True, fmt=".4f",annot_kws={'size':25})
In [ ]: